{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Compressive Transformer",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AYV_dMVDxyc2"
      },
      "source": [
        "[![Github](https://img.shields.io/github/stars/labmlai/annotated_deep_learning_paper_implementations?style=social)](https://github.com/labmlai/annotated_deep_learning_paper_implementations)\n",
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/labmlai/annotated_deep_learning_paper_implementations/blob/master/labml_nn/transformers/compressive/experiment.ipynb)                    \n",
        "\n",
        "## Compressive Transformer\n",
        "\n",
        "This is an experiment training Shakespeare dataset with a Compressive Transformer model."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AahG_i2y5tY9"
      },
      "source": [
        "Install the `labml-nn` package"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZCzmCrAIVg0L",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cf107fb2-4d50-4c67-af34-367624553421"
      },
      "source": [
        "!pip install labml-nn"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting labml-nn\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4c/22/1d55151f5b2ba65bf2241684a701ecbb5d3d88ba70a2761ac64a7a7d40ba/labml_nn-0.4.89-py3-none-any.whl (154kB)\n",
            "\r\u001b[K     |██▏                             | 10kB 25.8MB/s eta 0:00:01\r\u001b[K     |████▎                           | 20kB 32.1MB/s eta 0:00:01\r\u001b[K     |██████▍                         | 30kB 13.3MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 40kB 16.1MB/s eta 0:00:01\r\u001b[K     |██████████▋                     | 51kB 18.5MB/s eta 0:00:01\r\u001b[K     |████████████▊                   | 61kB 16.7MB/s eta 0:00:01\r\u001b[K     |██████████████▉                 | 71kB 16.7MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 81kB 18.2MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 92kB 17.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 102kB 16.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▎        | 112kB 16.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▍      | 122kB 16.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 133kB 16.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 143kB 16.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▊| 153kB 16.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 163kB 16.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from labml-nn) (1.19.5)\n",
            "Collecting einops\n",
            "  Downloading https://files.pythonhosted.org/packages/5d/a0/9935e030634bf60ecd572c775f64ace82ceddf2f504a5fd3902438f07090/einops-0.3.0-py2.py3-none-any.whl\n",
            "Collecting labml>=0.4.103\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/31/14/ffed9b2bd050230ee774ac178576da9152a753351a18e98e095128781af7/labml-0.4.103-py3-none-any.whl (101kB)\n",
            "\u001b[K     |████████████████████████████████| 102kB 12.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (from labml-nn) (1.7.0+cu101)\n",
            "Collecting labml-helpers>=0.4.76\n",
            "  Downloading https://files.pythonhosted.org/packages/49/df/4d920a4a221acd3cfa384dddb909ed0691b08682c0d8aeaabeee2138624f/labml_helpers-0.4.76-py3-none-any.whl\n",
            "Collecting gitpython\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fb/67/47a04d8a9d7f94645676fe683f1ee3fe9be01fe407686c180768a92abaac/GitPython-3.1.13-py3-none-any.whl (159kB)\n",
            "\u001b[K     |████████████████████████████████| 163kB 49.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from labml>=0.4.103->labml-nn) (3.13)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.6/dist-packages (from torch->labml-nn) (3.7.4.3)\n",
            "Requirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from torch->labml-nn) (0.8)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch->labml-nn) (0.16.0)\n",
            "Collecting gitdb<5,>=4.0.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/48/11/d1800bca0a3bae820b84b7d813ad1eff15a48a64caea9c823fc8c1b119e8/gitdb-4.0.5-py3-none-any.whl (63kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 12.3MB/s \n",
            "\u001b[?25hCollecting smmap<4,>=3.0.1\n",
            "  Downloading https://files.pythonhosted.org/packages/d5/1e/6130925131f639b2acde0f7f18b73e33ce082ff2d90783c436b52040af5a/smmap-3.0.5-py2.py3-none-any.whl\n",
            "Installing collected packages: einops, smmap, gitdb, gitpython, labml, labml-helpers, labml-nn\n",
            "Successfully installed einops-0.3.0 gitdb-4.0.5 gitpython-3.1.13 labml-0.4.103 labml-helpers-0.4.76 labml-nn-0.4.89 smmap-3.0.5\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SE2VUQ6L5zxI"
      },
      "source": [
        "Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0hJXx_g0wS2C"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "\n",
        "from labml import experiment\n",
        "from labml.configs import option\n",
        "from labml_nn.transformers.compressive.experiment import Configs"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lpggo0wM6qb-"
      },
      "source": [
        "Create an experiment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bFcr9k-l4cAg"
      },
      "source": [
        "experiment.create(name=\"compressive_transformer\")"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-OnHLi626tJt"
      },
      "source": [
        "Initialize configurations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Piz0c5f44hRo"
      },
      "source": [
        "conf = Configs()"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wwMzCqpD6vkL"
      },
      "source": [
        "Set experiment configurations and assign a configurations dictionary to override configurations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "e6hmQhTw4nks",
        "outputId": "29634715-42f4-4405-fb11-fc9522608627"
      },
      "source": [
        "experiment.configs(conf,\n",
        "                # A dictionary of configurations to override\n",
        "                {'tokenizer': 'character',\n",
        "                'text': 'tiny_shakespeare',\n",
        "                'optimizer.learning_rate': 2.5e-4,\n",
        "                'optimizer.optimizer': 'AdamW',\n",
        "                'prompt': 'It is',\n",
        "                'prompt_separator': '',\n",
        "\n",
        "                'train_loader': 'sequential_train_loader',\n",
        "                'valid_loader': 'sequential_valid_loader',\n",
        "\n",
        "                'seq_len': 8,\n",
        "                'mem_len': 8,\n",
        "                'epochs': 128,\n",
        "                'batch_size': 32,\n",
        "                'inner_iterations': 25,\n",
        "                'compression_rate': 2,\n",
        "                })"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<pre style=\"overflow-x: scroll;\"></pre>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EvI7MtgJ61w5"
      },
      "source": [
        "Set PyTorch models for loading and saving"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        },
        "id": "GDlt7dp-5ALt",
        "outputId": "e7548e8f-c541-4618-dc5a-1597cae42003"
      },
      "source": [
        "experiment.add_pytorch_models({'model': conf.model})"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<pre style=\"overflow-x: scroll;\">Prepare model...\n",
              "  Prepare n_tokens...\n",
              "    Prepare text...\n",
              "      Prepare tokenizer<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t4.14ms</span>\n",
              "      Download<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t106.70ms</span>\n",
              "      Load data<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t3.62ms</span>\n",
              "      Tokenize<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t30.35ms</span>\n",
              "      Build vocabulary<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t87.70ms</span>\n",
              "    Prepare text<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t246.32ms</span>\n",
              "  Prepare n_tokens<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t251.04ms</span>\n",
              "  Prepare device...\n",
              "    Prepare device_info<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t66.11ms</span>\n",
              "  Prepare device<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t69.06ms</span>\n",
              "Prepare model<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t10,529.11ms</span>\n",
              "</pre>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KJZRf8527GxL"
      },
      "source": [
        "Start the experiment and run the training loop."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "aIAWo7Fw5DR8",
        "outputId": "db979785-bfe3-4eda-d3eb-8ccbe61053e5"
      },
      "source": [
        "# Start the experiment\n",
        "with experiment.start():\n",
        "    conf.run()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<pre style=\"overflow-x: scroll;\">\n",
              "<strong><span style=\"text-decoration: underline\">compressive_transformer</span></strong>: <span style=\"color: #208FFB\">0d9b5338726c11ebb7c80242ac1c0002</span>\n",
              "\t[dirty]: <strong><span style=\"color: #DDB62B\">\"\"</span></strong>\n",
              "Initialize...\n",
              "  Prepare mode<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t5.53ms</span>\n",
              "Initialize<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t99.70ms</span>\n",
              "Prepare validator...\n",
              "  Prepare valid_loader<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t79.18ms</span>\n",
              "<span style=\"color: #C5C1B4\"></span>\n",
              "<span style=\"color: #C5C1B4\">--------------------------------------------------</span><span style=\"color: #DDB62B\"><strong><span style=\"text-decoration: underline\"></span></strong></span>\n",
              "<span style=\"color: #DDB62B\"><strong><span style=\"text-decoration: underline\">LABML WARNING</span></strong></span>\n",
              "<span style=\"color: #DDB62B\"><strong><span style=\"text-decoration: underline\"></span></strong></span>LabML App Warning: <span style=\"color: #60C6C8\">empty_token: </span><strong>Please create a valid token at https://app.labml.ai.</strong>\n",
              "<strong>Click on the experiment link to monitor the experiment and add it to your experiments list.</strong><span style=\"color: #C5C1B4\"></span>\n",
              "<span style=\"color: #C5C1B4\">--------------------------------------------------</span>\n",
              "Prepare validator<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t172.12ms</span>\n",
              "<span style=\"color: #208FFB\">Monitor experiment at </span><a href='https://app.labml.ai/run?uuid=0d9b5338726c11ebb7c80242ac1c0002' target='blank'>https://app.labml.ai/run?uuid=0d9b5338726c11ebb7c80242ac1c0002</a>\n",
              "Prepare trainer...\n",
              "  Prepare train_loader<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t148.07ms</span>\n",
              "Prepare trainer<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t187.12ms</span>\n",
              "Prepare training_loop...\n",
              "  Prepare loop_count<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t39.10ms</span>\n",
              "Prepare training_loop<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t302.88ms</span>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong>f</strong><strong>f</strong><strong>Q</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong><strong>S</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>f</strong><strong>f</strong><strong> </strong><strong>i</strong><strong>i</strong><strong>i</strong><strong>i</strong><strong>i</strong><strong>z</strong><strong> </strong><strong>f</strong><strong> </strong><strong>t</strong><strong> </strong><strong>t</strong><strong> </strong><strong>i</strong><strong>e</strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong>e</strong><strong> </strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong>r</strong><strong> </strong><strong>s</strong><strong>o</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong>r</strong><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>e</strong><strong>r</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong>r</strong><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>r</strong><strong>d</strong><strong>,</strong><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>f</strong><strong>o</strong><strong>r</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>u</strong><strong>g</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>g</strong><strong>o</strong><strong>o</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>a</strong><strong>v</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>d</strong><strong>e</strong><strong>a</strong><strong>t</strong><strong>h</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>d</strong><strong>e</strong><strong>a</strong><strong>t</strong><strong>h</strong><strong> </strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>a</strong><strong>v</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>a</strong><strong>v</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>f</strong><strong>o</strong><strong>r</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>f</strong><strong>o</strong><strong>r</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>f</strong><strong>o</strong><strong>r</strong><strong> </strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>i</strong><strong>s</strong><strong> </strong><strong>h</strong><strong>a</strong><strong>v</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>i</strong><strong>s</strong><strong> </strong><strong>h</strong><strong>a</strong><strong>v</strong>\n",
              "<strong><span style=\"color: #DDB62B\">1,003,776:  </span></strong>Sample:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\">   431ms  </span>Train:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\">    349,115ms  </span>Valid:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 13,573ms  </span> loss.train: <span style=\"color: #C5C1B4\"> 1.81316</span> accuracy.train: <span style=\"color: #C5C1B4\">0.352563</span> loss.valid: <span style=\"color: #C5C1B4\"> 1.85643</span> accuracy.valid: <span style=\"color: #C5C1B4\">0.345761</span>  <span style=\"color: #208FFB\">363,685ms</span><span style=\"color: #D160C4\">  0:06m/ 12:49m  </span>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>i</strong><strong>s</strong><strong> </strong><strong>f</strong><strong>o</strong><strong>r</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>f</strong><strong>a</strong><strong>i</strong><strong>r</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>a</strong><strong>v</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>f</strong><strong>a</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong>e</strong><strong>,</strong><strong></strong>\n",
              "<strong></strong><strong>A</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>i</strong><strong>t</strong><strong>h</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>n</strong><strong>e</strong><strong>s</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>i</strong><strong>s</strong><strong> </strong><strong>w</strong><strong>i</strong><strong>t</strong><strong>h</strong><strong> </strong><strong>t</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong></strong>\n",
              "<strong></strong><strong>T</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>f</strong><strong>o</strong><strong>r</strong><strong>t</strong><strong>h</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong></strong>\n",
              "<strong></strong><strong>T</strong><strong>h</strong><strong>a</strong><strong>t</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>o</strong><strong>u</strong><strong>l</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>a</strong><strong>t</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong></strong>\n",
              "<strong></strong><strong>T</strong><strong>h</strong><strong>a</strong><strong>t</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>i</strong><strong>t</strong><strong>h</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>i</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>o</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>o</strong><strong>n</strong><strong>.</strong><strong></strong>\n",
              "<strong></strong><strong></strong>\n",
              "<strong></strong><strong>L</strong><strong>O</strong><strong>U</strong><strong>C</strong><strong>E</strong><strong>S</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong></strong>\n",
              "<strong></strong><strong>A</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>i</strong><strong>t</strong><strong>h</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>i</strong><strong>t</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong></strong>\n",
              "<strong></strong><strong>T</strong><strong>h</strong><strong>a</strong><strong>t</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>o</strong><strong>r</strong><strong>n</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>a</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong></strong>\n",
              "<strong></strong><strong>I</strong><strong>n</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>o</strong><strong>r</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>o</strong><strong>r</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong></strong>\n",
              "<strong></strong><strong>T</strong><strong>h</strong><strong>i</strong><strong>s</strong><strong> </strong><strong>f</strong><strong>a</strong><strong>l</strong><strong>l</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>a</strong><strong>s</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>a</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong></strong>\n",
              "<strong></strong><strong>A</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>s</strong><strong>o</strong><strong> </strong><strong>f</strong><strong>a</strong><strong>i</strong><strong>m</strong><strong>,</strong><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>s</strong><strong>o</strong><strong> </strong><strong>f</strong><strong>a</strong><strong>i</strong><strong>r</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong></strong>\n",
              "<strong></strong><strong>T</strong><strong>h</strong><strong>a</strong><strong>t</strong><strong> </strong><strong>s</strong><strong>h</strong><strong>a</strong><strong>l</strong><strong>l</strong><strong> </strong><strong>b</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong></strong>\n",
              "<strong></strong><strong>T</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>a</strong><strong>n</strong><strong>d</strong><strong>s</strong><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>m</strong><strong>e</strong><strong>s</strong><strong>t</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong></strong>\n",
              "<strong></strong><strong>T</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>m</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>o</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>m</strong><strong>e</strong>\n",
              "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong>.</strong><strong></strong>\n",
              "<strong></strong><strong></strong>\n",
              "<strong></strong><strong>L</strong><strong>A</strong><strong>R</strong><strong>D</strong><strong>:</strong><strong></strong>\n",
              "<strong></strong><strong>M</strong><strong>y</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>r</strong><strong>r</strong><strong>o</strong><strong>w</strong><strong> </strong>\n",
              "<strong><span style=\"color: #DDB62B\">1,655,040:  </span></strong>Sample:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\">   451ms  </span>Train:<span style=\"color: #C5C1B4\">  64%</span><span style=\"color: #208FFB\">    363,339ms  </span>Valid:<span style=\"color: #C5C1B4\">  63%</span><span style=\"color: #208FFB\"> 15,211ms  </span> loss.train: <strong> 1.71129</strong> accuracy.train: <strong>0.495383</strong> loss.valid: <span style=\"color: #C5C1B4\"> 1.70167</span> accuracy.valid: <span style=\"color: #C5C1B4\">0.450568</span>  <span style=\"color: #208FFB\">363,685ms</span><span style=\"color: #D160C4\">  0:10m/ 12:45m  </span>\n",
              "Saving model...\n",
              "<span style=\"color: #E75C58\">Killing loop...</span>\n",
              "<strong><span style=\"color: #DDB62B\">Still updating LabML App, please wait for it to complete...</span></strong>\n",
              "<span style=\"color: #208FFB\">Updating App. Please wait...</span></pre>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-7-fb962b998049>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Start the experiment\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mexperiment\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mconf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/labml_helpers/train_valid.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    241\u001b[0m         \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    242\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_loop\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 243\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    244\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    245\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/labml_helpers/train_valid.py\u001b[0m in \u001b[0;36mrun_step\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    229\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_train\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    230\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0mtracker\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnamespace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'train'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 231\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    232\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0mtracker\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnamespace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'valid'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/labml_helpers/train_valid.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    133\u001b[0m                 \u001b[0msm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_epoch_start\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_grad_enabled\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 135\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__iterate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    136\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_index\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompleted\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/labml_helpers/train_valid.py\u001b[0m in \u001b[0;36m__iterate\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    146\u001b[0m                 \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__iterable\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 148\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    149\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    150\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_index\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/labml_nn/transformers/compressive/experiment.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, batch, batch_idx)\u001b[0m\n\u001b[1;32m    242\u001b[0m             \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip_grad_norm_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_norm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad_norm_clip\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    243\u001b[0m             \u001b[0;31m# Take optimizer step\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 244\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    245\u001b[0m             \u001b[0;31m# Log the model parameters and gradients on last batch of every epoch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    246\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_last\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/labml_nn/optimizers/__init__.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    155\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    156\u001b[0m                 \u001b[0;31m# Take the optimization step on the parameter\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 157\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep_param\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    158\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m         \u001b[0;31m# Return the loss, calculated from closure\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/labml_nn/optimizers/adam.py\u001b[0m in \u001b[0;36mstep_param\u001b[0;34m(self, state, group, grad, param)\u001b[0m\n\u001b[1;32m    206\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    207\u001b[0m         \u001b[0;31m# Get $m_t$ and $v_t$\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 208\u001b[0;31m         \u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_mv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    209\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m         \u001b[0;31m# Increment $t$ the number of optimizer steps\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/labml_nn/optimizers/amsgrad.py\u001b[0m in \u001b[0;36mget_mv\u001b[0;34m(self, state, group, grad)\u001b[0m\n\u001b[1;32m     83\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     84\u001b[0m         \u001b[0;31m# Get $m_t$ and $v_t$ from *Adam*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 85\u001b[0;31m         \u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_mv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     86\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m         \u001b[0;31m# If this parameter group is using `amsgrad`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/labml_nn/optimizers/adam.py\u001b[0m in \u001b[0;36mget_mv\u001b[0;34m(self, state, group, grad)\u001b[0m\n\u001b[1;32m    117\u001b[0m         \u001b[0;31m# In-place calculation of $v_t$\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m         \u001b[0;31m# $$v_t \\leftarrow \\beta_2 v_{t-1} + (1 - \\beta_2) \\cdot g_t^2$$\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 119\u001b[0;31m         \u001b[0mv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddcmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    120\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    121\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/labml_helpers/training_loop.py\u001b[0m in \u001b[0;36m__handler\u001b[0;34m(self, sig, frame)\u001b[0m\n\u001b[1;32m    162\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__finish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    163\u001b[0m             \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Killing loop...'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mText\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdanger\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 164\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mold_handler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mframe\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    165\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    166\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__str__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<pre style=\"overflow-x: scroll;\"></pre>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<pre style=\"overflow-x: scroll;\"><strong><span style=\"color: #DDB62B\">Finished updating LabML App.</span></strong></pre>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oBXXlP2b7XZO"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}